﻿using System.Collections.Generic;
using System.Text.RegularExpressions;
using Fetches.Fetch.Dto;
using Fetches.Fetch.Utils;

namespace Fetches.Fetch.Fetcher.LinkExtractor
{
    public class PatternMatchExtractor : ILinkExtractor
    {
        public List<Regex> IndexPatterns = new List<Regex>();
        public List<Regex> ContentPatterns = new List<Regex>();

        public PatternMatchExtractor AddIndexPatterns(IList<string> patterns)
        {
            foreach (var pattern in patterns)
            {
                IndexPatterns.Add(new Regex(pattern));
            }
            return this;
        }

        public PatternMatchExtractor AddContentPatterns(IList<string> patterns)
        {
            foreach (var pattern in patterns)
            {
                ContentPatterns.Add(new Regex(pattern));
            }

            return this;
        }

        #region 默认链接抽取器无需考虑额外的初始化和序列化

        public void Initialize(string config)
        {
            //
        }

        public string GetInitializeRule()
        {
            return string.Empty;
        }

        #endregion

        public IList<Request> ExtractLinks(ResponsePage responsePage)
        {
            var refer = responsePage.Request.Url.OriginalString;
            var result = new List<Request>();

            IList<KeyValuePair<string, string>> links = HtmlUtil.ExtractUrls(responsePage.SourceCode);
            var indexRequests = FilterLinks(links, refer, IndexPatterns);
            //Processor可能会根据这个属性来调整对页面的处理方式
            foreach (var indexRequest in indexRequests)
            {
                indexRequest.IndexPage = true;
            }


            result.AddRange(indexRequests);
            var contentRequests = FilterLinks(links, refer, ContentPatterns);
            result.AddRange(contentRequests);

            return result;
        }

        /// <summary>
        /// 将链接标准化并过滤,这个是用来避免爬取超出控制范围的
        /// </summary>
        /// <param name="links">链接列表key为url,value是文字</param>
        /// <param name="refer">当前页url</param>
        /// <param name="patterns">过滤标准</param>
        /// <returns></returns>
        protected IList<Request> FilterLinks(IList<KeyValuePair<string, string>> links, string refer, List<Regex> patterns = null)
        {
            var lst = new List<Request>();
            foreach (var item in links)
            {
                var url = item.Key;
                url = UrlUtils.NormalizingUrl(url, refer);
                bool include = true;
                if (patterns != null && patterns.Count > 0)
                {
                    include = false;
                    foreach (var pattern in patterns)
                    {
                        if (pattern.IsMatch(url))
                        {
                            include = true;
                            break;
                        }
                    }

                }
                if (include)
                {
                    lst.Add(new Request(url));
                }
            }
            return lst;
        }

    }
}
